{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import featuretools as ft\n",
    "from woodwork.logical_types import Categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customer_id</th>\n",
       "      <th>invoice</th>\n",
       "      <th>invoice_date</th>\n",
       "      <th>stock_code</th>\n",
       "      <th>description</th>\n",
       "      <th>quantity</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>2009-12-01 07:45:00</td>\n",
       "      <td>85048</td>\n",
       "      <td>15CM CHRISTMAS GLASS BALL 20 LIGHTS</td>\n",
       "      <td>12</td>\n",
       "      <td>6.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>2009-12-01 07:45:00</td>\n",
       "      <td>79323P</td>\n",
       "      <td>PINK CHERRY LIGHTS</td>\n",
       "      <td>12</td>\n",
       "      <td>6.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>2009-12-01 07:45:00</td>\n",
       "      <td>79323W</td>\n",
       "      <td>WHITE CHERRY LIGHTS</td>\n",
       "      <td>12</td>\n",
       "      <td>6.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>2009-12-01 07:45:00</td>\n",
       "      <td>22041</td>\n",
       "      <td>RECORD FRAME 7\" SINGLE SIZE</td>\n",
       "      <td>48</td>\n",
       "      <td>2.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>2009-12-01 07:45:00</td>\n",
       "      <td>21232</td>\n",
       "      <td>STRAWBERRY CERAMIC TRINKET BOX</td>\n",
       "      <td>24</td>\n",
       "      <td>1.25</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   customer_id invoice        invoice_date stock_code  \\\n",
       "0      13085.0  489434 2009-12-01 07:45:00      85048   \n",
       "1      13085.0  489434 2009-12-01 07:45:00     79323P   \n",
       "2      13085.0  489434 2009-12-01 07:45:00     79323W   \n",
       "3      13085.0  489434 2009-12-01 07:45:00      22041   \n",
       "4      13085.0  489434 2009-12-01 07:45:00      21232   \n",
       "\n",
       "                           description  quantity  price  \n",
       "0  15CM CHRISTMAS GLASS BALL 20 LIGHTS        12   6.95  \n",
       "1                   PINK CHERRY LIGHTS        12   6.75  \n",
       "2                  WHITE CHERRY LIGHTS        12   6.75  \n",
       "3         RECORD FRAME 7\" SINGLE SIZE         48   2.10  \n",
       "4       STRAWBERRY CERAMIC TRINKET BOX        24   1.25  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's load the data again\n",
    "\n",
    "df = pd.read_csv(\"retail.csv\", parse_dates=[\"invoice_date\"])\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create and entity set\n",
    "\n",
    "es = ft.EntitySet(id=\"data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Sole\\Documents\\Repositories\\envs\\fsml\\lib\\site-packages\\woodwork\\type_sys\\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.\n",
      "  pd.to_datetime(\n",
      "C:\\Users\\Sole\\Documents\\Repositories\\envs\\fsml\\lib\\site-packages\\woodwork\\type_sys\\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.\n",
      "  pd.to_datetime(\n",
      "C:\\Users\\Sole\\Documents\\Repositories\\envs\\fsml\\lib\\site-packages\\woodwork\\type_sys\\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.\n",
      "  pd.to_datetime(\n",
      "C:\\Users\\Sole\\Documents\\Repositories\\envs\\fsml\\lib\\site-packages\\woodwork\\type_sys\\utils.py:40: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.\n",
      "  pd.to_datetime(\n"
     ]
    }
   ],
   "source": [
    "# Add the data to the entity\n",
    "\n",
    "es = es.add_dataframe(\n",
    "    dataframe=df,              # the dataframe with the data\n",
    "    dataframe_name=\"data\",     # unique name to associate with this dataframe\n",
    "    index=\"rows\",              # column name to index the items\n",
    "    make_index=True,           # if true, create a new column with unique values\n",
    "    time_index=\"invoice_date\", # column containing time data\n",
    "    logical_types={\n",
    "        \"customer_id\": Categorical, # the id is numerical, but should be handled as categorical\n",
    "        \"invoice\": Categorical,\n",
    "    },\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Physical Type</th>\n",
       "      <th>Logical Type</th>\n",
       "      <th>Semantic Tag(s)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Column</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rows</th>\n",
       "      <td>int64</td>\n",
       "      <td>Integer</td>\n",
       "      <td>['index']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>customer_id</th>\n",
       "      <td>category</td>\n",
       "      <td>Categorical</td>\n",
       "      <td>['category']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>invoice</th>\n",
       "      <td>category</td>\n",
       "      <td>Categorical</td>\n",
       "      <td>['category']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>invoice_date</th>\n",
       "      <td>datetime64[ns]</td>\n",
       "      <td>Datetime</td>\n",
       "      <td>['time_index']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stock_code</th>\n",
       "      <td>category</td>\n",
       "      <td>Categorical</td>\n",
       "      <td>['category']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>description</th>\n",
       "      <td>category</td>\n",
       "      <td>Categorical</td>\n",
       "      <td>['category']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>quantity</th>\n",
       "      <td>int64</td>\n",
       "      <td>Integer</td>\n",
       "      <td>['numeric']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>price</th>\n",
       "      <td>float64</td>\n",
       "      <td>Double</td>\n",
       "      <td>['numeric']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "               Physical Type Logical Type Semantic Tag(s)\n",
       "Column                                                   \n",
       "rows                   int64      Integer       ['index']\n",
       "customer_id         category  Categorical    ['category']\n",
       "invoice             category  Categorical    ['category']\n",
       "invoice_date  datetime64[ns]     Datetime  ['time_index']\n",
       "stock_code          category  Categorical    ['category']\n",
       "description         category  Categorical    ['category']\n",
       "quantity               int64      Integer     ['numeric']\n",
       "price                float64       Double     ['numeric']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es[\"data\"].ww"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Entityset: data\n",
       "  DataFrames:\n",
       "    data [Rows: 741301, Columns: 8]\n",
       "    invoices [Rows: 40505, Columns: 3]\n",
       "  Relationships:\n",
       "    data.invoice -> invoices.invoice"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a new dataframe with invoices\n",
    "# indicating its relationship to the main data\n",
    "\n",
    "es.normalize_dataframe(\n",
    "    base_dataframe_name=\"data\",     # Datarame name from which to split.\n",
    "    new_dataframe_name=\"invoices\",  # Name of the new dataframe.\n",
    "    index=\"invoice\",                # relationship will be created across this column.\n",
    "    copy_columns=[\"customer_id\"],   # columns to remove from base_dataframe and move to new dataframe.\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rows</th>\n",
       "      <th>customer_id</th>\n",
       "      <th>invoice</th>\n",
       "      <th>invoice_date</th>\n",
       "      <th>stock_code</th>\n",
       "      <th>description</th>\n",
       "      <th>quantity</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>2009-12-01 07:45:00</td>\n",
       "      <td>85048</td>\n",
       "      <td>15CM CHRISTMAS GLASS BALL 20 LIGHTS</td>\n",
       "      <td>12</td>\n",
       "      <td>6.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>2009-12-01 07:45:00</td>\n",
       "      <td>79323P</td>\n",
       "      <td>PINK CHERRY LIGHTS</td>\n",
       "      <td>12</td>\n",
       "      <td>6.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>2009-12-01 07:45:00</td>\n",
       "      <td>79323W</td>\n",
       "      <td>WHITE CHERRY LIGHTS</td>\n",
       "      <td>12</td>\n",
       "      <td>6.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>2009-12-01 07:45:00</td>\n",
       "      <td>22041</td>\n",
       "      <td>RECORD FRAME 7\" SINGLE SIZE</td>\n",
       "      <td>48</td>\n",
       "      <td>2.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>2009-12-01 07:45:00</td>\n",
       "      <td>21232</td>\n",
       "      <td>STRAWBERRY CERAMIC TRINKET BOX</td>\n",
       "      <td>24</td>\n",
       "      <td>1.25</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   rows customer_id invoice        invoice_date stock_code  \\\n",
       "0     0     13085.0  489434 2009-12-01 07:45:00      85048   \n",
       "1     1     13085.0  489434 2009-12-01 07:45:00     79323P   \n",
       "2     2     13085.0  489434 2009-12-01 07:45:00     79323W   \n",
       "3     3     13085.0  489434 2009-12-01 07:45:00      22041   \n",
       "4     4     13085.0  489434 2009-12-01 07:45:00      21232   \n",
       "\n",
       "                           description  quantity  price  \n",
       "0  15CM CHRISTMAS GLASS BALL 20 LIGHTS        12   6.95  \n",
       "1                   PINK CHERRY LIGHTS        12   6.75  \n",
       "2                  WHITE CHERRY LIGHTS        12   6.75  \n",
       "3         RECORD FRAME 7\" SINGLE SIZE         48   2.10  \n",
       "4       STRAWBERRY CERAMIC TRINKET BOX        24   1.25  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the original data\n",
    "\n",
    "es[\"data\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cumulative transform primitives\n",
    "\n",
    "cum_primitives = [\"cum_sum\", \"cum_max\", \"diff\", \"time_since_previous\"]\n",
    "\n",
    "general_primitives = [\"sine\", \"cosine\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<Feature: customer_id>,\n",
       " <Feature: invoice>,\n",
       " <Feature: stock_code>,\n",
       " <Feature: description>,\n",
       " <Feature: quantity>,\n",
       " <Feature: price>,\n",
       " <Feature: COSINE(price)>,\n",
       " <Feature: COSINE(quantity)>,\n",
       " <Feature: SINE(price)>,\n",
       " <Feature: SINE(quantity)>,\n",
       " <Feature: CUM_MAX(price) by invoice>,\n",
       " <Feature: CUM_MAX(quantity) by invoice>,\n",
       " <Feature: CUM_SUM(price) by invoice>,\n",
       " <Feature: CUM_SUM(quantity) by invoice>,\n",
       " <Feature: DIFF(price) by invoice>,\n",
       " <Feature: DIFF(quantity) by invoice>,\n",
       " <Feature: TIME_SINCE_PREVIOUS(invoice_date) by invoice>]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# if you want to double check the result of the\n",
    "# feature creation process, set features_only=True\n",
    "\n",
    "feature_defs = ft.dfs(\n",
    "    entityset=es,                                # the entity set\n",
    "    target_dataframe_name=\"data\",                # the dataframe for wich to create the features\n",
    "    agg_primitives=[],                           # empty list to avoid returning the defo parameters\n",
    "    trans_primitives=general_primitives,         # empty list to avoid returning the defo parameters\n",
    "    groupby_trans_primitives = cum_primitives,   # the operations to perform by invoice\n",
    "    ignore_dataframes = [\"invoices\"],            # columns to ignore when creating features\n",
    "    features_only=True,    \n",
    ")\n",
    "\n",
    "# display name of created features\n",
    "feature_defs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Sole\\Documents\\Repositories\\envs\\fsml\\lib\\site-packages\\featuretools\\computational_backends\\feature_set_calculator.py:545: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  grouped = frame.groupby(groupby)\n",
      "C:\\Users\\Sole\\Documents\\Repositories\\envs\\fsml\\lib\\site-packages\\featuretools\\computational_backends\\feature_set_calculator.py:588: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method.\n",
      "The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy.\n",
      "\n",
      "For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.\n",
      "\n",
      "\n",
      "  frame[name].update(pd.concat(col_vals))\n",
      "C:\\Users\\Sole\\Documents\\Repositories\\envs\\fsml\\lib\\site-packages\\featuretools\\computational_backends\\feature_set_calculator.py:545: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  grouped = frame.groupby(groupby)\n",
      "C:\\Users\\Sole\\Documents\\Repositories\\envs\\fsml\\lib\\site-packages\\featuretools\\computational_backends\\feature_set_calculator.py:588: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method.\n",
      "The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy.\n",
      "\n",
      "For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.\n",
      "\n",
      "\n",
      "  frame[name].update(pd.concat(col_vals))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<Feature: customer_id>,\n",
       " <Feature: invoice>,\n",
       " <Feature: stock_code>,\n",
       " <Feature: description>,\n",
       " <Feature: quantity>,\n",
       " <Feature: price>,\n",
       " <Feature: COSINE(price)>,\n",
       " <Feature: COSINE(quantity)>,\n",
       " <Feature: SINE(price)>,\n",
       " <Feature: SINE(quantity)>,\n",
       " <Feature: CUM_MAX(price) by invoice>,\n",
       " <Feature: CUM_MAX(quantity) by invoice>,\n",
       " <Feature: CUM_SUM(price) by invoice>,\n",
       " <Feature: CUM_SUM(quantity) by invoice>,\n",
       " <Feature: DIFF(price) by invoice>,\n",
       " <Feature: DIFF(quantity) by invoice>,\n",
       " <Feature: TIME_SINCE_PREVIOUS(invoice_date) by invoice>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create all features simultaneously\n",
    "\n",
    "feature_matrix, feature_defs = ft.dfs(\n",
    "    entityset=es,                                # the entity set\n",
    "    target_dataframe_name=\"data\",                # the dataframe for wich to create the features\n",
    "    agg_primitives=[],                           # empty list to avoid returning the defo parameters\n",
    "    trans_primitives=general_primitives,         # empty list to avoid returning the defo parameters\n",
    "    groupby_trans_primitives = cum_primitives,   # the operations to perform by invoice\n",
    "    ignore_dataframes = [\"invoices\"],            # columns to ignore when creating features\n",
    ")\n",
    "\n",
    "# display name of created features\n",
    "feature_defs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customer_id</th>\n",
       "      <th>invoice</th>\n",
       "      <th>stock_code</th>\n",
       "      <th>description</th>\n",
       "      <th>quantity</th>\n",
       "      <th>price</th>\n",
       "      <th>COSINE(price)</th>\n",
       "      <th>COSINE(quantity)</th>\n",
       "      <th>SINE(price)</th>\n",
       "      <th>SINE(quantity)</th>\n",
       "      <th>CUM_MAX(price) by invoice</th>\n",
       "      <th>CUM_MAX(quantity) by invoice</th>\n",
       "      <th>CUM_SUM(price) by invoice</th>\n",
       "      <th>CUM_SUM(quantity) by invoice</th>\n",
       "      <th>DIFF(price) by invoice</th>\n",
       "      <th>DIFF(quantity) by invoice</th>\n",
       "      <th>TIME_SINCE_PREVIOUS(invoice_date) by invoice</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rows</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>85048</td>\n",
       "      <td>15CM CHRISTMAS GLASS BALL 20 LIGHTS</td>\n",
       "      <td>12</td>\n",
       "      <td>6.95</td>\n",
       "      <td>0.785796</td>\n",
       "      <td>0.843854</td>\n",
       "      <td>0.618486</td>\n",
       "      <td>-0.536573</td>\n",
       "      <td>6.95</td>\n",
       "      <td>12.0</td>\n",
       "      <td>6.95</td>\n",
       "      <td>12.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>79323P</td>\n",
       "      <td>PINK CHERRY LIGHTS</td>\n",
       "      <td>12</td>\n",
       "      <td>6.75</td>\n",
       "      <td>0.893006</td>\n",
       "      <td>0.843854</td>\n",
       "      <td>0.450044</td>\n",
       "      <td>-0.536573</td>\n",
       "      <td>6.95</td>\n",
       "      <td>12.0</td>\n",
       "      <td>13.70</td>\n",
       "      <td>24.0</td>\n",
       "      <td>-0.20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>79323W</td>\n",
       "      <td>WHITE CHERRY LIGHTS</td>\n",
       "      <td>12</td>\n",
       "      <td>6.75</td>\n",
       "      <td>0.893006</td>\n",
       "      <td>0.843854</td>\n",
       "      <td>0.450044</td>\n",
       "      <td>-0.536573</td>\n",
       "      <td>6.95</td>\n",
       "      <td>12.0</td>\n",
       "      <td>20.45</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>22041</td>\n",
       "      <td>RECORD FRAME 7\" SINGLE SIZE</td>\n",
       "      <td>48</td>\n",
       "      <td>2.10</td>\n",
       "      <td>-0.504846</td>\n",
       "      <td>-0.640144</td>\n",
       "      <td>0.863209</td>\n",
       "      <td>-0.768255</td>\n",
       "      <td>6.95</td>\n",
       "      <td>48.0</td>\n",
       "      <td>22.55</td>\n",
       "      <td>84.0</td>\n",
       "      <td>-4.65</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>21232</td>\n",
       "      <td>STRAWBERRY CERAMIC TRINKET BOX</td>\n",
       "      <td>24</td>\n",
       "      <td>1.25</td>\n",
       "      <td>0.315322</td>\n",
       "      <td>0.424179</td>\n",
       "      <td>0.948985</td>\n",
       "      <td>-0.905578</td>\n",
       "      <td>6.95</td>\n",
       "      <td>48.0</td>\n",
       "      <td>23.80</td>\n",
       "      <td>108.0</td>\n",
       "      <td>-0.85</td>\n",
       "      <td>-24.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     customer_id invoice stock_code                          description  \\\n",
       "rows                                                                       \n",
       "0        13085.0  489434      85048  15CM CHRISTMAS GLASS BALL 20 LIGHTS   \n",
       "1        13085.0  489434     79323P                   PINK CHERRY LIGHTS   \n",
       "2        13085.0  489434     79323W                  WHITE CHERRY LIGHTS   \n",
       "3        13085.0  489434      22041         RECORD FRAME 7\" SINGLE SIZE    \n",
       "4        13085.0  489434      21232       STRAWBERRY CERAMIC TRINKET BOX   \n",
       "\n",
       "      quantity  price  COSINE(price)  COSINE(quantity)  SINE(price)  \\\n",
       "rows                                                                  \n",
       "0           12   6.95       0.785796          0.843854     0.618486   \n",
       "1           12   6.75       0.893006          0.843854     0.450044   \n",
       "2           12   6.75       0.893006          0.843854     0.450044   \n",
       "3           48   2.10      -0.504846         -0.640144     0.863209   \n",
       "4           24   1.25       0.315322          0.424179     0.948985   \n",
       "\n",
       "      SINE(quantity)  CUM_MAX(price) by invoice  CUM_MAX(quantity) by invoice  \\\n",
       "rows                                                                            \n",
       "0          -0.536573                       6.95                          12.0   \n",
       "1          -0.536573                       6.95                          12.0   \n",
       "2          -0.536573                       6.95                          12.0   \n",
       "3          -0.768255                       6.95                          48.0   \n",
       "4          -0.905578                       6.95                          48.0   \n",
       "\n",
       "      CUM_SUM(price) by invoice  CUM_SUM(quantity) by invoice  \\\n",
       "rows                                                            \n",
       "0                          6.95                          12.0   \n",
       "1                         13.70                          24.0   \n",
       "2                         20.45                          36.0   \n",
       "3                         22.55                          84.0   \n",
       "4                         23.80                         108.0   \n",
       "\n",
       "      DIFF(price) by invoice  DIFF(quantity) by invoice  \\\n",
       "rows                                                      \n",
       "0                        NaN                        NaN   \n",
       "1                      -0.20                        0.0   \n",
       "2                       0.00                        0.0   \n",
       "3                      -4.65                       36.0   \n",
       "4                      -0.85                      -24.0   \n",
       "\n",
       "      TIME_SINCE_PREVIOUS(invoice_date) by invoice  \n",
       "rows                                                \n",
       "0                                              NaN  \n",
       "1                                              0.0  \n",
       "2                                              0.0  \n",
       "3                                              0.0  \n",
       "4                                              0.0  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Resulting datatable with original \n",
    "# and new features\n",
    "\n",
    "feature_matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(741301, 17)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_matrix.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## In relation to pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customer_id</th>\n",
       "      <th>invoice</th>\n",
       "      <th>invoice_date</th>\n",
       "      <th>stock_code</th>\n",
       "      <th>description</th>\n",
       "      <th>quantity</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>2009-12-01 07:45:00</td>\n",
       "      <td>85048</td>\n",
       "      <td>15CM CHRISTMAS GLASS BALL 20 LIGHTS</td>\n",
       "      <td>12</td>\n",
       "      <td>6.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>2009-12-01 07:45:00</td>\n",
       "      <td>79323P</td>\n",
       "      <td>PINK CHERRY LIGHTS</td>\n",
       "      <td>12</td>\n",
       "      <td>6.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>2009-12-01 07:45:00</td>\n",
       "      <td>79323W</td>\n",
       "      <td>WHITE CHERRY LIGHTS</td>\n",
       "      <td>12</td>\n",
       "      <td>6.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>2009-12-01 07:45:00</td>\n",
       "      <td>22041</td>\n",
       "      <td>RECORD FRAME 7\" SINGLE SIZE</td>\n",
       "      <td>48</td>\n",
       "      <td>2.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13085.0</td>\n",
       "      <td>489434</td>\n",
       "      <td>2009-12-01 07:45:00</td>\n",
       "      <td>21232</td>\n",
       "      <td>STRAWBERRY CERAMIC TRINKET BOX</td>\n",
       "      <td>24</td>\n",
       "      <td>1.25</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   customer_id invoice        invoice_date stock_code  \\\n",
       "0      13085.0  489434 2009-12-01 07:45:00      85048   \n",
       "1      13085.0  489434 2009-12-01 07:45:00     79323P   \n",
       "2      13085.0  489434 2009-12-01 07:45:00     79323W   \n",
       "3      13085.0  489434 2009-12-01 07:45:00      22041   \n",
       "4      13085.0  489434 2009-12-01 07:45:00      21232   \n",
       "\n",
       "                           description  quantity  price  \n",
       "0  15CM CHRISTMAS GLASS BALL 20 LIGHTS        12   6.95  \n",
       "1                   PINK CHERRY LIGHTS        12   6.75  \n",
       "2                  WHITE CHERRY LIGHTS        12   6.75  \n",
       "3         RECORD FRAME 7\" SINGLE SIZE         48   2.10  \n",
       "4       STRAWBERRY CERAMIC TRINKET BOX        24   1.25  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load data\n",
    "\n",
    "df = pd.read_csv(\"retail.csv\", parse_dates=[\"invoice_date\"])\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['quantity_cumsum',\n",
       " 'price_cumsum',\n",
       " 'quantity_cummax',\n",
       " 'price_cummax',\n",
       " 'quantity_diff',\n",
       " 'price_diff']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Cumulative sum at invoice level\n",
    "\n",
    "# numerical variables\n",
    "numeric_vars = [\"quantity\", \"price\"]\n",
    "\n",
    "# the cumulative functions\n",
    "func = [\"cumsum\", \"cummax\", \"diff\"]\n",
    "\n",
    "# new variable names\n",
    "new_names = [f\"{var}_{function}\" for function in func for var in numeric_vars]\n",
    "\n",
    "new_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[new_names] = df.groupby(\n",
    "    \"invoice\")[numeric_vars].agg(func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>quantity</th>\n",
       "      <th>price</th>\n",
       "      <th>quantity_cumsum</th>\n",
       "      <th>price_cumsum</th>\n",
       "      <th>quantity_cummax</th>\n",
       "      <th>price_cummax</th>\n",
       "      <th>quantity_diff</th>\n",
       "      <th>price_diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12</td>\n",
       "      <td>6.95</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.95</td>\n",
       "      <td>6.95</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12</td>\n",
       "      <td>6.75</td>\n",
       "      <td>24</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.70</td>\n",
       "      <td>6.95</td>\n",
       "      <td>-0.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12</td>\n",
       "      <td>6.75</td>\n",
       "      <td>36</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.45</td>\n",
       "      <td>6.95</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>48</td>\n",
       "      <td>2.10</td>\n",
       "      <td>84</td>\n",
       "      <td>48</td>\n",
       "      <td>36.0</td>\n",
       "      <td>22.55</td>\n",
       "      <td>6.95</td>\n",
       "      <td>-4.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>24</td>\n",
       "      <td>1.25</td>\n",
       "      <td>108</td>\n",
       "      <td>48</td>\n",
       "      <td>-24.0</td>\n",
       "      <td>23.80</td>\n",
       "      <td>6.95</td>\n",
       "      <td>-0.85</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   quantity  price  quantity_cumsum  price_cumsum  quantity_cummax  \\\n",
       "0        12   6.95               12            12              NaN   \n",
       "1        12   6.75               24            12              0.0   \n",
       "2        12   6.75               36            12              0.0   \n",
       "3        48   2.10               84            48             36.0   \n",
       "4        24   1.25              108            48            -24.0   \n",
       "\n",
       "   price_cummax  quantity_diff  price_diff  \n",
       "0          6.95           6.95         NaN  \n",
       "1         13.70           6.95       -0.20  \n",
       "2         20.45           6.95        0.00  \n",
       "3         22.55           6.95       -4.65  \n",
       "4         23.80           6.95       -0.85  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# inspect values for 1 invoice\n",
    "\n",
    "df[df[\"invoice\"] == \"489434\" ][numeric_vars + new_names].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['quantity_sin', 'price_sin', 'quantity_cos', 'price_cos']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# general transformations\n",
    "\n",
    "new_names = [f\"{var}_{function}\" for function in [\"sin\", \"cos\"]\n",
    "             for var in numeric_vars]\n",
    "\n",
    "new_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Sole\\AppData\\Local\\Temp\\ipykernel_6776\\866113479.py:1: FutureWarning: using <ufunc 'sin'> in Series.agg cannot aggregate and has been deprecated. Use Series.transform to keep behavior unchanged.\n",
      "  df[new_names] = df[numeric_vars].agg([np.sin, np.cos])\n",
      "C:\\Users\\Sole\\AppData\\Local\\Temp\\ipykernel_6776\\866113479.py:1: FutureWarning: using <ufunc 'cos'> in Series.agg cannot aggregate and has been deprecated. Use Series.transform to keep behavior unchanged.\n",
      "  df[new_names] = df[numeric_vars].agg([np.sin, np.cos])\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>quantity_sin</th>\n",
       "      <th>price_sin</th>\n",
       "      <th>quantity_cos</th>\n",
       "      <th>price_cos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.536573</td>\n",
       "      <td>0.843854</td>\n",
       "      <td>0.618486</td>\n",
       "      <td>0.785796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.536573</td>\n",
       "      <td>0.843854</td>\n",
       "      <td>0.450044</td>\n",
       "      <td>0.893006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.536573</td>\n",
       "      <td>0.843854</td>\n",
       "      <td>0.450044</td>\n",
       "      <td>0.893006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.768255</td>\n",
       "      <td>-0.640144</td>\n",
       "      <td>0.863209</td>\n",
       "      <td>-0.504846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.905578</td>\n",
       "      <td>0.424179</td>\n",
       "      <td>0.948985</td>\n",
       "      <td>0.315322</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   quantity_sin  price_sin  quantity_cos  price_cos\n",
       "0     -0.536573   0.843854      0.618486   0.785796\n",
       "1     -0.536573   0.843854      0.450044   0.893006\n",
       "2     -0.536573   0.843854      0.450044   0.893006\n",
       "3     -0.768255  -0.640144      0.863209  -0.504846\n",
       "4     -0.905578   0.424179      0.948985   0.315322"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[new_names] = df[numeric_vars].agg([np.sin, np.cos])\n",
    "\n",
    "df[new_names].head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fsml",
   "language": "python",
   "name": "fsml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
